{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-25T04:04:46.700393Z",
     "start_time": "2020-03-25T04:04:46.695376Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from itertools import combinations\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "MAX_ITERATION = 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-25T04:04:10.800935Z",
     "start_time": "2020-03-25T04:04:10.765008Z"
    }
   },
   "outputs": [],
   "source": [
    "def data_generator():\n",
    "    iris = pd.read_csv('iris.csv', usecols=[1, 2, 3, 4, 5])\n",
    "    X = np.array(\n",
    "        iris[['Sepal.Length', 'Sepal.Width', 'Petal.Length', 'Petal.Width']])\n",
    "    y = iris[['Species']].copy()\n",
    "    y[y['Species'] == 'setosa'] = 0\n",
    "    y[y['Species'] == 'versicolor'] = 1\n",
    "    y[y['Species'] == 'virginica'] = 2\n",
    "    y = np.array(y).reshape(-1)\n",
    "\n",
    "    X0_train, X0_test, y0_train, y0_test = train_test_split(X[np.where(y == 0)],\n",
    "                                                            y[np.where(\n",
    "                                                                y == 0)],\n",
    "                                                            test_size=20, random_state=1)\n",
    "    X1_train, X1_test, y1_train, y1_test = train_test_split(X[np.where(y == 1)],\n",
    "                                                            y[np.where(\n",
    "                                                                y == 1)],\n",
    "                                                            test_size=20, random_state=2)\n",
    "    X2_train, X2_test, y2_train, y2_test = train_test_split(X[np.where(y == 2)],\n",
    "                                                            y[np.where(\n",
    "                                                                y == 2)],\n",
    "                                                            test_size=20, random_state=3)\n",
    "    X_train = np.concatenate((X0_train, X1_train, X2_train), axis=0)\n",
    "    y_train = np.concatenate((y0_train, y1_train, y2_train))\n",
    "    X_test = np.concatenate((X0_test, X1_test, X2_test), axis=0)\n",
    "    y_test = np.concatenate((y0_test, y1_test, y2_test))\n",
    "    X_train, y_train = shuffle(X_train, y_train)\n",
    "    X_test, y_test = shuffle(X_test, y_test)\n",
    "\n",
    "    return X_train, y_train, X_test, y_test\n",
    "\n",
    "\n",
    "def OVO_data_preprocessing(X_train, y_train, X_test, y_test, num_classes):\n",
    "    def data_slice(X, y, combination):\n",
    "        X1 = X[np.where(y == combination[0])]\n",
    "        X2 = X[np.where(y == combination[1])]\n",
    "        X_ = np.concatenate((X1, X2), axis=0)\n",
    "        y_ = np.concatenate((np.full(len((X1)), -1), np.full(len((X2)), 1)))\n",
    "        X_, y_ = shuffle(X_, y_)\n",
    "\n",
    "        return X_, y_\n",
    "\n",
    "    PLA_X_train = []\n",
    "    PLA_X_test = []\n",
    "    PLA_y_train = []\n",
    "\n",
    "    PLAs_combination = list(combinations(range(num_classes), 2))\n",
    "    for i in range(len(PLAs_combination)):\n",
    "        _X_train, _y_train = data_slice(X_train, y_train, PLAs_combination[i])\n",
    "        sc = StandardScaler()\n",
    "        sc.fit(_X_train)\n",
    "        _X_train = sc.transform(_X_train)\n",
    "        _X_test = sc.transform(X_test)\n",
    "        PLA_X_train.append(_X_train)\n",
    "        PLA_X_test.append(_X_test)\n",
    "        PLA_y_train.append(_y_train)\n",
    "\n",
    "    return PLA_X_train, PLA_y_train, PLA_X_test\n",
    "\n",
    "\n",
    "def PLA(X, y, w_init, max_iter, lr=1):\n",
    "    w = w_init.copy()\n",
    "    X = np.concatenate((np.ones((X.shape[0], 1)), X), axis=1)\n",
    "\n",
    "    for i in range(max_iter):\n",
    "        h = np.sign(np.dot(X, w))\n",
    "        mistake_indices = np.where(h != y)[0]\n",
    "        mistake_num = len(mistake_indices)\n",
    "        if mistake_num > 0:\n",
    "            np.random.shuffle(mistake_indices)\n",
    "            j = np.random.choice(mistake_indices)\n",
    "            w_ = w + lr * y[j]*X[j, :]\n",
    "            h_ = np.sign(np.dot(X, w_))\n",
    "            mistake_indices_ = np.where(h_ != y)[0]\n",
    "            mistake_num_ = len(mistake_indices_)\n",
    "            if mistake_num_ <= mistake_num:\n",
    "                w = w_\n",
    "        else:\n",
    "            break\n",
    "\n",
    "    h = np.sign(np.dot(X, w))\n",
    "    mistake_indices = np.where(h != y)[0]\n",
    "    mistake_num = len(mistake_indices)\n",
    "    print('Accuracy of Perceptron Learning Algorithm is %.2f%%' %\n",
    "          (100*(X.shape[0]-mistake_num)/X.shape[0]))\n",
    "    return w\n",
    "\n",
    "\n",
    "def PLA_OVO_train(X, y, num_classes):\n",
    "    w = []\n",
    "    PLAs_combination = list(combinations(range(num_classes), 2))\n",
    "    for i in range(len(PLAs_combination)):\n",
    "        np.random.seed()\n",
    "        w_init = np.random.randn(X[i].shape[1]+1)\n",
    "        w.append(PLA(X[i], y[i], w_init, MAX_ITERATION, lr=0.05))\n",
    "\n",
    "    return np.array(w)\n",
    "\n",
    "\n",
    "def PLA_OVO_test(X, y, num_classes, w):\n",
    "    votes = []\n",
    "    PLAs_combination = list(combinations(range(num_classes), 2))\n",
    "    for i in range(len(PLAs_combination)):\n",
    "        h = np.sign(np.dot(np.concatenate(\n",
    "            (np.ones((X[i].shape[0], 1)), X[i]), axis=1), w[i]))\n",
    "        h = np.where(h == -1, PLAs_combination[i][0], PLAs_combination[i][1])\n",
    "        votes.append(h)\n",
    "    votes = np.array(votes).T\n",
    "    y_pred = np.apply_along_axis(lambda x: np.argmax(np.bincount(x)), 1, votes)\n",
    "\n",
    "    mistake_indices = np.where(y != y_pred)[0]\n",
    "    mistake_num = len(mistake_indices)\n",
    "    print('Accuracy of PLA_OVO is %.2f%%' %\n",
    "          (100*(X[0].shape[0]-mistake_num)/X[0].shape[0]))\n",
    "    print('y_pred:', y_pred)\n",
    "    print('y_true:', y)\n",
    "\n",
    "\n",
    "def softmax_train(X, y, num_classes, max_iter, lr=0.001, eps=1e-4):\n",
    "    def softmax(X, w):\n",
    "        scores = np.dot(X, w.T)\n",
    "        c = np.max(scores, axis=1).reshape(-1, 1)\n",
    "        P = np.exp(scores - c) / \\\n",
    "            np.sum(np.exp(scores - c), axis=1).reshape(-1, 1)\n",
    "        return P\n",
    "\n",
    "    np.random.seed()\n",
    "    w = np.random.randn(num_classes, X.shape[1]+1)\n",
    "    X = np.concatenate((np.ones((X.shape[0], 1)), X), axis=1)\n",
    "    Y = np.eye(num_classes)[y]\n",
    "\n",
    "    for i in range(max_iter):\n",
    "        P = softmax(X, w)\n",
    "        diff = -1/X.shape[0] * np.dot((Y-P).T, X)\n",
    "        w_ = w - lr * diff\n",
    "        delta = np.linalg.norm(w - w_)\n",
    "        if (np.argmax(P, axis=1) == y).all():\n",
    "            break\n",
    "        if delta < eps:\n",
    "            break\n",
    "        else:\n",
    "            w = w_\n",
    "    print('Final weight matrix is:', w)\n",
    "    return w\n",
    "\n",
    "\n",
    "def softmax_test(X, y, num_classes, w):\n",
    "    def softmax(X, w):\n",
    "        scores = np.dot(X, w.T)\n",
    "        c = np.max(scores, axis=1).reshape(-1, 1)\n",
    "        P = np.exp(scores - c) / \\\n",
    "            np.sum(np.exp(scores - c), axis=1).reshape(-1, 1)\n",
    "        return P\n",
    "\n",
    "    X = np.concatenate((np.ones((X.shape[0], 1)), X), axis=1)\n",
    "    P = softmax(X, w)\n",
    "    y_pred = np.argmax(P, axis=1)\n",
    "    mistake_indices = np.where(y_pred != y)[0]\n",
    "    mistake_num = len(mistake_indices)\n",
    "    print('Accuracy of Softmax is %.2f%%' %\n",
    "          (100*(X.shape[0]-mistake_num)/X.shape[0]))\n",
    "    print('y_pred:', y_pred)\n",
    "    print('y_true:', y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PLA OVO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-25T04:05:01.997377Z",
     "start_time": "2020-03-25T04:05:01.490415Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of Perceptron Learning Algorithm is 100.00%\n",
      "Accuracy of Perceptron Learning Algorithm is 100.00%\n",
      "Accuracy of Perceptron Learning Algorithm is 98.33%\n",
      "Accuracy of PLA_OVO is 95.00%\n",
      "y_pred: [2 0 0 2 2 1 1 1 1 2 1 1 1 0 2 1 2 0 0 2 2 0 1 1 2 0 0 0 2 0 0 2 2 0 1 2 2\n",
      " 2 2 0 2 1 1 2 0 0 1 0 2 1 0 1 0 1 2 0 0 2 1 1]\n",
      "y_true: [2 0 0 2 2 2 1 1 1 2 1 1 1 0 2 1 2 0 0 2 2 0 1 1 2 0 0 0 2 0 0 2 1 0 1 1 2\n",
      " 2 2 0 2 1 1 2 0 0 1 0 2 1 0 1 0 1 2 0 0 2 1 1]\n"
     ]
    }
   ],
   "source": [
    "X_train, y_train, X_test, y_test = data_generator()\n",
    "PLA_X_train, PLA_y_train, PLA_X_test = OVO_data_preprocessing(X_train, y_train, X_test, y_test, 3)\n",
    "w = PLA_OVO_train(PLA_X_train, PLA_y_train, 3)\n",
    "PLA_OVO_test(PLA_X_test, y_test, 3, w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Softmax Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-03-24T07:39:52.631953Z",
     "start_time": "2020-03-24T07:39:49.958807Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final weight matrix is: [[ 2.14084355e+00  2.87979283e+00  4.75036492e+00 -7.27068917e+00\n",
      "   2.52720899e-01]\n",
      " [ 4.25455078e+00  2.27654969e+00 -5.68400397e-03 -9.40890519e-01\n",
      "  -2.81530083e+00]\n",
      " [-5.12112404e+00 -3.32147360e+00 -5.81262040e+00  8.14678338e+00\n",
      "   7.05984124e+00]]\n",
      "Accuracy of Softmax is 98.33%\n",
      "y_pred: [2 0 0 1 1 2 0 2 0 1 2 0 2 1 0 0 1 1 2 2 0 2 1 0 0 0 1 2 1 0 0 1 2 2 2 2 2\n",
      " 0 2 1 2 1 0 2 0 2 1 2 0 1 0 2 1 0 1 0 1 1 1 2]\n",
      "y_true: [2 0 0 1 1 2 0 2 0 1 2 0 2 1 0 0 1 1 2 1 0 2 1 0 0 0 1 2 1 0 0 1 2 2 2 2 2\n",
      " 0 2 1 2 1 0 2 0 2 1 2 0 1 0 2 1 0 1 0 1 1 1 2]\n"
     ]
    }
   ],
   "source": [
    "w = softmax_train(X_train, y_train, 3, MAX_ITERATION, lr=0.05)\n",
    "softmax_test(X_test, y_test, 3, w)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
